from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
from fake_useragent import UserAgent
from openpyxl import load_workbook
from openpyxl.styles import Font, Alignment, Border, Side, PatternFill
import pandas as pd
import logging
import time
import os


# 日志配置
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')


# 常量定义
FILE_PATH = os.path.dirname(os.path.abspath(__file__))
CONFIG_FILE = os.path.join(FILE_PATH, 'config.txt')
ALIYUN_URL = 'https://www.aliyun.com/search?k={}&scene=all'
OUTPUT_FILE = os.path.join(FILE_PATH, 'aliyun.xlsx')
SCM_SELECTOR = '//*[@class="mobileList"]/div[@class="item"]'
TITLE_SELECTOR = '.title'
CONTENT_SELECTOR = '.content'
NODES_INFO_SELECTOR = '.nodes-info'
HREF_SELECTOR = './/*[@class="ace-link ace-link-primary "]'
ua = UserAgent()
user_agent = ua.random

# 配置Chrome选项
OPTIONS = Options()
OPTIONS.add_argument('--log-level=3')                                       # 设置日志级别为3
OPTIONS.add_argument(f'--user-agent={user_agent}')                          # 添加User-Agent
OPTIONS.add_argument('--ignore-certificate-errors')                         # 忽略证书错误
OPTIONS.add_argument('--ignore-ssl-errors')                                 # 忽略SSL错误
OPTIONS.add_experimental_option('excludeSwitches', ['enable-automation'])   # 禁用自动化扩展
OPTIONS.add_experimental_option('useAutomationExtension', False)            # 禁用自动化扩展
OPTIONS.add_argument('--disable-blink-features=AutomationControlled')       # 禁用Blink运行时的自动化控制
OPTIONS.add_argument('--headless')                                          # 无头模式, 不显示浏览器窗口

# 初始化Driver
DRIVER = webdriver.Chrome(options=OPTIONS)
DRIVER.implicitly_wait(5)

def read_config():
    """读取配置文件"""
    if os.path.exists(CONFIG_FILE):
        with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
            return f.read().strip()
    return None

def write_config(data_name):
    """写入配置文件"""
    with open(CONFIG_FILE, 'w', encoding='utf-8') as f:
        f.write(data_name)

def get_data_name():
    """获取资源名称"""
    data_name = input('请输入想要获取的资源名称: ')
    if data_name:
        write_config(data_name)
    else:
        data_name = read_config()
    return data_name

def open_url(data_name):
    """打开指定URL"""
    url = ALIYUN_URL.format(data_name)
    logging.info(f'正在请求目标网址: {url}')
    DRIVER.get(url)
    try:
        WebDriverWait(DRIVER, 5).until(
            EC.presence_of_element_located((By.XPATH, '//div[@class="ace-aliyun-search-result"]'))
        )
        logging.info('页面加载成功')
    except Exception as e:
        logging.error('页面加载失败:', e)
        DRIVER.quit()
        exit()


def scroll_to_bottom_and_wait(expected_count=None):
    """滚动到底部并等待新内容加载到预期数量"""
    last_message = ""
    max_retries = 3
    base_timeout = 5  # 基础超时时间
    timeout_increment = 10  # 每次重试增加的超时时间
    current_count = len(DRIVER.find_elements(By.XPATH, SCM_SELECTOR))   #初始化,用于给 prev_count 赋值
    current_timeout = base_timeout + (3 - max_retries) * timeout_increment
    while max_retries > 0:
        prev_count = current_count
        DRIVER.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        try:
            WebDriverWait(DRIVER, current_timeout).until(
                lambda driver: len(driver.find_elements(By.XPATH, SCM_SELECTOR)) > prev_count
            )
            current_count = len(DRIVER.find_elements(By.XPATH, SCM_SELECTOR))   # 此处重新查询元素长度,能够第一时间获取更新后的数据条数
            last_message = f'{'-' * 10}> 已加载 {current_count} 条数据 <{'-' * 10}'
            print(last_message, end='\r')

            if expected_count and current_count >= expected_count:
                last_message = f'成功加载 {current_count} 条数据'
                logging.info(last_message)
                return True
        except TimeoutException:
            max_retries -= 1
            last_message = f'等待新内容加载超时，正在尝试重新获取数据，已加载 {prev_count} 条数据'
            logging.warning(last_message)
            while True:
                try:
                    current_count = len(DRIVER.find_elements(By.XPATH, SCM_SELECTOR))
                    restrat_message = f'prev_count: {prev_count} / current_count: {current_count}'
                    print(restrat_message, end='\r')
                    if current_count >= prev_count:
                        logging.info('已跟踪到数据上次加载的位置')
                        break
                    WebDriverWait(DRIVER, 5).until(
                        lambda driver: len(driver.find_elements(By.XPATH, SCM_SELECTOR)) >= prev_count
                    )
                except TimeoutException:
                    time.sleep(2)
            if max_retries == 0:
                time.sleep(2)
                current_count = len(DRIVER.find_elements(By.XPATH, SCM_SELECTOR))
                last_message = f'已加载 {current_count} 条数据，无更多数据可加载'
                logging.warning(last_message)
                return False


def extract_data(elements):
    """提取数据"""
    logging.info('数据正在格式化中')
    data = []
    for index, element in enumerate(elements, start=1):
        title = element.find_element(By.CSS_SELECTOR, TITLE_SELECTOR).text
        content = element.find_element(By.CSS_SELECTOR, CONTENT_SELECTOR).text
        nodes_info = element.find_element(By.CSS_SELECTOR, NODES_INFO_SELECTOR).text
        link_element = element.find_element(By.XPATH, HREF_SELECTOR)
        href = link_element.get_attribute('href')
        data.append({'序号': index, '标题': title, '链接': href, '内容': content, '来自': nodes_info})
    return data

def save_data_to_excel(data):
    """保存数据到Excel文件"""
    df_data = pd.DataFrame(data)
    df_data.to_excel(OUTPUT_FILE, index=False)
        # 加载工作簿
    wb = load_workbook(OUTPUT_FILE)
    ws = wb.active

    # 设置列宽
    column_widths = [10, 30, 50, 80, 20]  # 自定义每列宽度
    for i, width in enumerate(column_widths, start=1):
        ws.column_dimensions[chr(64 + i)].width = width

    # 设置首行标题样式
    header_fill = PatternFill(start_color='000000', end_color='000000', fill_type='solid')
    header_font = Font(color='FFFFFF', bold=True)
    header_alignment = Alignment(horizontal='center', vertical='center')
    for cell in ws[1]:
        cell.fill = header_fill
        cell.font = header_font
        cell.alignment = header_alignment

    # 设置单元格边框样式
    thin_border = Border(left=Side(style='thin', color='000000'),
                         right=Side(style='thin', color='000000'),
                         top=Side(style='thin', color='000000'),
                         bottom=Side(style='thin', color='000000'))
    for row in ws.iter_rows(min_row=1, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
        for cell in row:
            cell.border = thin_border

    # 保存修改后的工作簿
    wb.save(OUTPUT_FILE)
    logging.info('数据保存成功')

def run_script():
    """运行脚本"""
    data_name = get_data_name()
    open_url(data_name)

    # all_elements = []

    logging.info('请确认爬取数量, 回车默认爬取全部数据')
    page_number = input('请输入: ')

    if page_number:
        try:
            page_number = int(page_number)
            if scroll_to_bottom_and_wait(page_number):
                all_elements = DRIVER.find_elements(By.XPATH, SCM_SELECTOR)[:page_number]
                logging.info(f'成功获取 {len(all_elements)} 条数据')
            else:
                all_elements = DRIVER.find_elements(By.XPATH, SCM_SELECTOR)
                logging.warning(f'未能获取到 {page_number} 条数据，已获取 {len(all_elements)} 条数据')
        except ValueError:
            logging.error('输入的不是有效的数字，请输入整数。')
            return
    else:
        logging.info('未输入，正在爬取全部数据')
        if not scroll_to_bottom_and_wait():
            all_elements = DRIVER.find_elements(By.XPATH, SCM_SELECTOR)
            logging.info(f'成功获取 {len(all_elements)} 条数据')
        else:
            logging.warning('未查询到数据')

    data = extract_data(all_elements)
    save_data_to_excel(data)

def main():
    """主函数"""
    while True:
        run_script()
        # print("你好，未来！")
        choice = input("请选择（1.继续 2.退出）: ").strip()
        if choice == '1':
            continue  # 退出循环
        elif choice == '2':
            break  # 退出循环
        else:
            while choice!= '1' and choice!= '2':
                print("无效的选择，请重新输入。")
                choice = input("请选择（1.继续 2.退出）: ").strip()
    DRIVER.quit() # 退出驱动程序

if __name__ == '__main__':
    main()
